﻿using System;
using System.Collections.Generic;
using System.Text;

namespace DiscoveryLogic.TextMining.Tokenization
{
    public class Tokenizer:ITokenizer
    {
        #region fields
        private char[] stream;
        private int curPos;
        private bool intQFlg;
        private int len;
        private bool ignoreDelim;
        private bool xml;
        private bool inXmlTag;
        public static string sentenceDelimSet = "\"<";
        public static string delimSet = " \n\t\r,.;:!?&()<>[]+\"'";
        public static string whiteSpace = " \n\t\r";
        public static string digitSet = "0123456789";
        #endregion

        #region ctor
        public Tokenizer()
        {
            this.stream = null;
            this.curPos = 0;
            this.intQFlg = false;
            this.len = 0;
            this.ignoreDelim = false;
            this.xml = false;
            this.inXmlTag = false;
        }

        public Tokenizer(string s)
        {
            this.stream = s.ToCharArray();
            this.curPos = 0;
            this.intQFlg = false;
            this.len = s.Length;
            this.ignoreDelim = false;
            this.xml = false;
            this.inXmlTag = false;
        }
        #endregion

        #region props
        public bool IgnoreDelim
        {
            set { ignoreDelim = value; }
        }

        public bool IsXml
        {
            set { xml = value; }
        }
        #endregion

        #region ITokenizer Members

        public List<string> TokenizeDocument(string content)
        {
            List<string> tokens = new List<string>();
            if(!string.IsNullOrEmpty(content))
            {
                this.reset(content);
                string word = this.nextToken();
                while (word!=null)
                {
                    tokens.Add(word);
                    word = this.nextToken();
                }
            }
            return tokens;
        }

        #endregion

        #region util
        public void reset(string s)
        {
            stream = s.ToCharArray();
            curPos = 0;
            intQFlg = false;
            inXmlTag = false;
            len = s.Length;
        }

        public string nextToken()
        {
            string s3;

            int i = 0;
            bool flag = false;
            do
            {
                if (!flag)
                {
                    i = curPos;
                    if (i >= len)
                        return null;
                }
                else
                {
                    flag = false;
                }
                for (; i < len && (inXmlTag && stream[i] != '>' || delimSet.IndexOf(stream[i]) < 0); i++)
                {
                }
                if (i == len)
                {
                    if (i == curPos)
                        return null;
                }
                else
                {
                    if (whiteSpace.IndexOf(stream[i]) >= 0)
                    {
                        if (i == curPos)
                        {
                            curPos++;
                        }
                        else
                        {
                            string s = this.CopyValueOf(stream, curPos, i - curPos);
                            curPos = i + 1;
                            return s;
                        }
                        continue;
                    }
                    if (stream[i] == '\'')
                    {
                        if (i == 0 || delimSet.IndexOf(stream[i - 1]) >= 0)
                        {
                            intQFlg = true;
                            curPos++;
                        }
                        else if (i < len - 1 && delimSet.IndexOf(stream[i + 1]) < 0)
                        {
                            i++;
                            flag = true;
                        }
                        else
                        {
                            string s1;
                            if (intQFlg)
                            {
                                intQFlg = false;
                                s1 = this.CopyValueOf(stream, curPos, i - curPos);
                            }
                            else
                            {
                                s1 = this.CopyValueOf(stream, curPos, (i - curPos) + 1);
                            }
                            curPos = i + 1;
                            return s1;
                        }
                        continue;
                    }
                    if (stream[i] == ':')
                    {
                        if (i < len - 1 && delimSet.IndexOf(stream[i + 1]) < 0)
                        {
                            i++;
                            flag = true;
                            continue;
                        }
                    }
                    else if (stream[i] == ',')
                    {
                        if (i < len - 1 && digitSet.IndexOf(stream[i + 1]) >= 0)
                        {
                            i++;
                            flag = true;
                            continue;
                        }
                    }
                    else if (stream[i] == '.')
                    {
                        if (!IsEndOfSentence(i))
                        {
                            i++;
                            flag = true;
                            continue;
                        }
                    }
                    else if (xml && stream[i] == '<')
                    {
                        if (!inXmlTag && i == curPos && whiteSpace.IndexOf(stream[i + 1]) < 0)
                        {
                            i++;
                            flag = true;
                            inXmlTag = true;
                            continue;
                        }
                    }
                    else if (xml && stream[i] == '>' && inXmlTag && i > 0 && whiteSpace.IndexOf(stream[i - 1]) < 0)
                    {
                        string s2 = this.CopyValueOf(stream, curPos, (i - curPos) + 1);
                        curPos = i + 1;
                        inXmlTag = false;
                        return s2;
                    }
                }
                if (curPos != i)
                    break;
                curPos++;
                if (!ignoreDelim)
                {
                    s3 = stream[i].ToString();
                    break;
                }
            } while (true);
            s3 = this.CopyValueOf(stream, curPos, i - curPos);
            if (ignoreDelim && (i >= len || stream[i] != '<'))
                curPos = i + 1;
            else
                curPos = i;

            return s3;
        }

        private string CopyValueOf(char[] charArray, int pos, int length)
        {
            StringBuilder buffer=new StringBuilder();
            for (int i = pos; i < pos + length; i++)
            {
                buffer.Append(charArray[i]);
            }
            return buffer.ToString();
        }

        private bool IsEndOfSentence(int i)
        {
            if (i == len - 1)
                return true;
            int j;
            for (j = i + 1; j < len && whiteSpace.IndexOf(stream[j]) >= 0; j++)
            {
            }
            if (j == len)
                return true;
            if (i == curPos && (whiteSpace.IndexOf(stream[i + 1]) >= 0 || j == i + 1))
                return true;
            if (i - curPos < 2)
                return false;
            if (i < 3)
                return false;
            if (sentenceDelimSet != null && sentenceDelimSet.IndexOf(stream[i + 1]) >= 0)
                return true;
            if (whiteSpace.IndexOf(stream[i + 1]) < 0)
                return false;
            if (whiteSpace.IndexOf(stream[i - 1]) >= 0)
                return false;
            if ("\"')}]".IndexOf(stream[i - 1]) >= 0)
                return true;
            int k;
            for (k = curPos; k < i && stream[k] != '.'; k++)
            {
            }
            if (k < i)
                return false;
            if (i - curPos < 5 && char.IsUpper(stream[curPos]) && char.IsUpper(stream[j]))
                return false;
            if (char.IsUpper(stream[j]))
                return true;
            if (char.IsDigit(stream[j]))
                return true;
            return "$<({[\"'".IndexOf(stream[j]) >= 0;
        }
        #endregion
    }
}
